import requests
from lxml import etree
import pymongo
from pymongo.collection import Collection

class DangDang(object):
    #链接数据库
    mongo_client=pymongo_client=pymongo.MongoClient(host="127.0.0.1",port=27017)
    # 指定数据库
    dangdang_db = mongo_client["dangdang_db"]

    def __init__(self):
        self.headers = {
            'Host': 'bang.dangdang.com',
            'connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9'
        }
        self.dang = Collection(DangDang.dangdang_db,"dangdang")
    def get_url(self, page):
        url = 'http://bang.dangdang.com/books/fivestars/1-%s'%page

        response = requests.get(url=url, headers=self.headers)
        if response:
            html = etree.HTML(response.text)
            m='//ul[@class="bang_list clearfix bang_list_mode"]//li'
            iterms = html.xpath(m)
            return iterms

    def join_list(self,it):
        #处理字符串
        return "".join(it)

    def pare_iterms(self,iterms):
        book_list=[]
        for iterm in iterms:
            name = iterm.xpath('.//div[@class="name"]/a/text()')#书名
            star = iterm.xpath('.//div[@class="star"]/a/text()')#评论
            star_tuijian = iterm.xpath('.//div[@class="star"]/span[@class="tuijian"]/text()') #推荐
            publisher = iterm.xpath('.//div[@class="publisher_info"][1]/a[1]/text()') #作者
            publisher_time = iterm.xpath('.//div[@class="publisher_info"][2]/span/text()')  #出版时间
            publisher_info = iterm.xpath('.//div[@class="publisher_info"][2]/a/text()')       #出版社
            pingfen = iterm.xpath('.//div[@class="biaosheng"]//text() ')     #评分
            price_n = iterm.xpath('.//div[@class="price"]/p[1]/span[@class="price_n"]/text()')  #价格
            price_s = iterm.xpath('.//div[@class="price"]//span[@class="price_s"]/text() ')  #折扣
            price_e = iterm.xpath('.//div[@class="price"]/p[@class="price_e"]/span/text()')   #电子书价格

            #把每一本书的信息封装成字典，在把书放入列表
            book_list.append(
                {
                    "书名": self.join_list(name),
                    "评论": self.join_list(star),
                    "推荐": self.join_list(star_tuijian),
                    "作者": self.join_list(publisher),
                    "出版时间": self.join_list(publisher_time),
                    "出版社": self.join_list(publisher_info),
                    "评分": self.join_list(pingfen),
                    "价格": self.join_list(price_n),
                    "折扣": self.join_list(price_s),
                    "电子书价格":self.join_list(price_e)
                }
            )
        return book_list

    def insert_data(self,book_list):
         self.dang.insert_many(book_list)



def main():
    d = DangDang()
    for page in range(1, 26):
        iterm = d.get_url(page=page)
        book_list =d.pare_iterms(iterms=iterm)
        print(book_list)
        d.insert_data(book_list)


if __name__ == '__main__':
    main()

